# 爬取诗词名句网三国演义所有章节和所有章节内容
import requests
from bs4 import BeautifulSoup

if __name__ == '__main__':
    # 请求头
    headers = {
        'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36"
    }
    url = 'https://www.shicimingju.com/book/sanguoyanyi.html'
    page_text = requests.get(url=url, headers=headers).text
    # 在首页解析出章节的标题和详情页url
    soup = BeautifulSoup(page_text, 'lxml')
    # 解析章节标题和详情页url
    li_list = soup.select('.book-mulu>ul>li')
    with open('./sanguo.txt', 'w', encoding='utf-8') as fp:
        for li in li_list:
            title = li.a.text
            detail_url = 'https://shicimingju.com' + li.a['href']
            # 对详情页发起请求，解析出章节内容
            detail_page_text = requests.get(url=detail_url, headers=headers).text
            # 解析出详情页中的相关的章节内容
            detail_soup = BeautifulSoup(detail_page_text, 'lxml')
            div_tag = detail_soup.find('div', class_='chapter_content')
            content = div_tag.text
            fp.write(title + ":" + content + '\n')
            print(title,'爬取成功!!!')
